# https://www.kuaidaili.com/free/inha/
"""
       快代理爬虫
       :return:
       """
import time

import requests
from bs4 import BeautifulSoup
from lxml import etree

headers={
        'Connection': 'keep-alive',
        # 'Cache-Control': 'max-age=0',
        # 'Accept': 'text/html, */*; q=0.01',
        # 'X-Requested-With': 'XMLHttpRequest',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36',
        # 'DNT': '1',
        # 'Accept-Encoding': 'gzip, deflate, sdch',
        # 'Accept-Language': 'zh-CN,zh;q=0.8,ja;q=0.6',
    }
Proxies = []
for i in range(2,6):
    url = "https://www.kuaidaili.com/free/inha/{}/".format(i)
    print(url)
    time.sleep(3)
    res = requests.get(url, headers=headers)
    html = BeautifulSoup(res.text, 'html.parser')
    div = html.find('div',attrs={'id':'list'})
    table = div.find('table').find('tbody')
    trs = table.find_all('tr')
    Proxies = []
    for tr in trs:
        # td = tr.find_all('td')
        # print(td)
        if tr:
            # print(tr)
            ip = tr.find('td',attrs={'data-title':'IP'}).text
            port = tr.find('td',attrs={'data-title':'PORT'}).text
            anonymous = 1 if '匿' in tr.find('td',attrs={'data-title':'匿名度'}).text else 0
            isHttps = 1 if  tr.find('td',attrs={'data-title':'类型'}).text=='HTTPS' else 0
            verifyTime = tr.find('td',attrs={'data-title':'最后验证时间'}).text
            Proxies.append({'ip':ip,'port':port,'anonymous':anonymous,'isHttps':isHttps,'verifyTime':verifyTime})
    # try:
    #     et = etree.HTML(res.text)
    #     ip_list = et.xpath('//*[@data-title="IP"]/text()')
    #     port_list = et.xpath('//*[@data-title="PORT"]/text()')
    #     isHttps = et.xpath('//*[@data-title="类型"]/text()')
    #     anonymous = et.xpath('//*[@data-title="匿名度"]/text()')
    #     verifyTime = et.xpath('//*[@data-title="最后验证时间"]/text()')
    #     for ip, port in zip(ip_list, port_list):
    #         print( ip + ":" + port)
    # except Exception as e:
    #     print(e)

